# common libraries
from datetime import datetime
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
import geopandas as gpd
%matplotlib inline

# metrics
from sklearn.metrics import davies_bouldin_score

# scalers
from sklearn.preprocessing import StandardScaler

# unsupervised learning (clustering) libraries
from sklearn.cluster import AgglomerativeClustering, KMeans, DBSCAN
from yellowbrick.cluster import KElbowVisualizer
from scipy.cluster.hierarchy import linkage,dendrogram
from sklearn.neighbors import NearestNeighbors

# PCA libraries
from sklearn.decomposition import PCA
from yellowbrick.features import PCA as PCA_viz

# Import state abbreviations and FIPS
from state_names import state_abbreviations, state_to_fips

# read in the dataset
data = pd.read_csv("all_data.csv", index_col=0)
data.head()

# 'Value' column is actually average temperature value so let's change its name
data = data.rename(columns={'Value': 'Average Temperature Value'})

#mapping states by FIPS Code
data['StateFPS'] = data['StateAbbrv'].map(state_to_fips)

# Loading the shapefile in order to visualize clean data on the US map
us_counties = gpd.read_file('tl_2022_us_county/tl_2022_us_county.shp')
print(us_counties.columns)
us_counties.head()

Index(['STATEFP', 'COUNTYFP', 'COUNTYNS', 'GEOID', 'NAME', 'NAMELSAD', 'LSAD',
       'CLASSFP', 'MTFCC', 'CSAFP', 'CBSAFP', 'METDIVFP', 'FUNCSTAT', 'ALAND',
       'AWATER', 'INTPTLAT', 'INTPTLON', 'geometry'],
      dtype='object')

# Merge the geospatial United States map with the data
data = us_counties.merge(data, left_on=['NAMELSAD', 'STATEFP'], right_on=['RegionName', 'StateFPS'])

data.head()

#checking for missing values
data.isnull().sum()

STATEFP                                         0
COUNTYFP                                        0
COUNTYNS                                        0
GEOID                                           0
NAME                                            0
NAMELSAD                                        0
LSAD                                            0
CLASSFP                                         0
MTFCC                                           0
CSAFP                                      579894
CBSAFP                                     579894
METDIVFP                                   579894
FUNCSTAT                                        0
ALAND                                           0
AWATER                                          0
INTPTLAT                                        0
INTPTLON                                        0
geometry                                        0
Date                                            0
Year                                            0
Month                                           0
RegionName                                      0
State                                           0
Bottom-Tier Average Home Value Estimate         0
Top-Tier Average Home Value Estimate            0
Average Temperature Value                       0
Anomaly (1901-2000 base period)                 0
1901-2000 Mean                                  0
StateAbbrv                                      0
Initial Fees and Charges (%)                    0
Effective Rate (%)                              0
Term to Maturity                                0
Loan Amount ($thou)                             0
Purchase Price ($thou)                          0
Loan-to-Price Ratio (%)                         0
Share of Total Market (%)                       0
StateFPS                                        0
dtype: int64

# dropping the columns with NaN because they are not necessary for mapping
data =  data.drop(columns = ['CSAFP', 'CBSAFP','METDIVFP'])

# check the types of all columns
data.dtypes

STATEFP                                      object
COUNTYFP                                     object
COUNTYNS                                     object
GEOID                                        object
NAME                                         object
NAMELSAD                                     object
LSAD                                         object
CLASSFP                                      object
MTFCC                                        object
FUNCSTAT                                     object
ALAND                                         int64
AWATER                                        int64
INTPTLAT                                     object
INTPTLON                                     object
geometry                                   geometry
Date                                         object
Year                                          int64
Month                                         int64
RegionName                                   object
State                                        object
Bottom-Tier Average Home Value Estimate     float64
Top-Tier Average Home Value Estimate        float64
Average Temperature Value                   float64
Anomaly (1901-2000 base period)             float64
1901-2000 Mean                              float64
StateAbbrv                                   object
Initial Fees and Charges (%)                float64
Effective Rate (%)                          float64
Term to Maturity                              int64
Loan Amount ($thou)                         float64
Purchase Price ($thou)                      float64
Loan-to-Price Ratio (%)                     float64
Share of Total Market (%)                   float64
StateFPS                                     object
dtype: object

# convert the 'Date' column to datetime format
data['Date']= pd.to_datetime(data['Date'])
 
# Check the format of 'Date' column
data.dtypes

STATEFP                                            object
COUNTYFP                                           object
COUNTYNS                                           object
GEOID                                              object
NAME                                               object
NAMELSAD                                           object
LSAD                                               object
CLASSFP                                            object
MTFCC                                              object
FUNCSTAT                                           object
ALAND                                               int64
AWATER                                              int64
INTPTLAT                                           object
INTPTLON                                           object
geometry                                         geometry
Date                                       datetime64[ns]
Year                                                int64
Month                                               int64
RegionName                                         object
State                                              object
Bottom-Tier Average Home Value Estimate           float64
Top-Tier Average Home Value Estimate              float64
Average Temperature Value                         float64
Anomaly (1901-2000 base period)                   float64
1901-2000 Mean                                    float64
StateAbbrv                                         object
Initial Fees and Charges (%)                      float64
Effective Rate (%)                                float64
Term to Maturity                                    int64
Loan Amount ($thou)                               float64
Purchase Price ($thou)                            float64
Loan-to-Price Ratio (%)                           float64
Share of Total Market (%)                         float64
StateFPS                                           object
dtype: object

#this is the list of columns we are interested in because most of them are numeric columns with information that is valuable for our project
data.columns[data.columns.get_loc('ALAND'):-1]

Index(['ALAND', 'AWATER', 'INTPTLAT', 'INTPTLON', 'geometry', 'Date', 'Year',
       'Month', 'RegionName', 'State',
       'Bottom-Tier Average Home Value Estimate',
       'Top-Tier Average Home Value Estimate', 'Average Temperature Value',
       'Anomaly (1901-2000 base period)', '1901-2000 Mean', 'StateAbbrv',
       'Initial Fees and Charges (%)', 'Effective Rate (%)',
       'Term to Maturity', 'Loan Amount ($thou)', 'Purchase Price ($thou)',
       'Loan-to-Price Ratio (%)', 'Share of Total Market (%)'],
      dtype='object')

# let's get the three columns we not want in the list above out to get our unsupervised dataset that contains numerical data
unsupervised = data[data.columns[data.columns.get_loc('ALAND'):-1].difference(['geometry','Date','StateAbbrv','RegionName','State'])] 
unsupervised.head(5)

# check the types again
unsupervised.dtypes

1901-2000 Mean                             float64
ALAND                                        int64
AWATER                                       int64
Anomaly (1901-2000 base period)            float64
Average Temperature Value                  float64
Bottom-Tier Average Home Value Estimate    float64
Effective Rate (%)                         float64
INTPTLAT                                    object
INTPTLON                                    object
Initial Fees and Charges (%)               float64
Loan Amount ($thou)                        float64
Loan-to-Price Ratio (%)                    float64
Month                                        int64
Purchase Price ($thou)                     float64
Share of Total Market (%)                  float64
Term to Maturity                             int64
Top-Tier Average Home Value Estimate       float64
Year                                         int64
dtype: object

# change the latitude and longtitude data to float type
unsupervised = unsupervised.astype({
    'INTPTLAT':'float', 
    'INTPTLON':'float'
})
unsupervised.dtypes

1901-2000 Mean                             float64
ALAND                                        int64
AWATER                                       int64
Anomaly (1901-2000 base period)            float64
Average Temperature Value                  float64
Bottom-Tier Average Home Value Estimate    float64
Effective Rate (%)                         float64
INTPTLAT                                   float64
INTPTLON                                   float64
Initial Fees and Charges (%)               float64
Loan Amount ($thou)                        float64
Loan-to-Price Ratio (%)                    float64
Month                                        int64
Purchase Price ($thou)                     float64
Share of Total Market (%)                  float64
Term to Maturity                             int64
Top-Tier Average Home Value Estimate       float64
Year                                         int64
dtype: object

# check the shape of unsupervised
unsupervised.shape

(579894, 18)

# double check NaN values one more time
unsupervised.isnull().sum()

1901-2000 Mean                             0
ALAND                                      0
AWATER                                     0
Anomaly (1901-2000 base period)            0
Average Temperature Value                  0
Bottom-Tier Average Home Value Estimate    0
Effective Rate (%)                         0
INTPTLAT                                   0
INTPTLON                                   0
Initial Fees and Charges (%)               0
Loan Amount ($thou)                        0
Loan-to-Price Ratio (%)                    0
Month                                      0
Purchase Price ($thou)                     0
Share of Total Market (%)                  0
Term to Maturity                           0
Top-Tier Average Home Value Estimate       0
Year                                       0
dtype: int64

# have an overview of our dataset
unsupervised.describe()

# double check colinearity through a heatmap
plt.figure(figsize=(15,15))

sns.heatmap(
    #get the correlation
    unsupervised.corr(),

    #annotate heat map
    annot=True,
    fmt ='.2f', 

    #mask half of the corr matrix
    mask = np.triu(unsupervised.corr()) 
)

#turn off plt grid and show group
plt.grid(False)
plt.show()

# visualize the clusters through a pairplot of a smaller sample of the dataset (it would take too long to plot the entire dataset)
sns.pairplot(
    unsupervised.sample(500))
plt.xticks(rotation = 45)
plt.show()

# we have a lot of data and one of our previous visualization shows seasonality so let us check that for housing prices and average temperature

# plotting two sublpots of average housing prices over time and average temperature over time to check for seasonality 
fig, ax = plt.subplots(3,1, figsize=(17,15))
cols = ['Average Temperature Value', 'Purchase Price ($thou)', 'Anomaly (1901-2000 base period)']

for i in range(3):
    sns.lineplot(
        data = data, #let's use the original dataset because it has a 'Date' column
        x = 'Date', 
        y = cols[i], 
        ax= ax[i]
    ).set(
        title = cols[i]+ ' Averaged across Counties in the US from 2000 to 2019'
        )
    ax[i].grid(False)

#show the plots
plt.show()

# plotting average housing prices for two years (2016, 2017) to have a better view of seasonality potentials
fig, ax = plt.subplots(2,1, figsize=(17,15))

year = [2016, 2017]

for i in range(2):

    sns.lineplot(
        data = data[data['Year']==year[i]], 
        x = 'Month', 
        y = 'Purchase Price ($thou)', 
        ax=ax[i]
    ).set(title= 'Purchase Price in '+ str(year[i]))

plt.show()

# dropping columns with high correlations with an existing datatype and dropping categorical data
cols = ['Month', 'Year', 'Term to Maturity']
unsupervised = unsupervised.drop(columns=cols)

#replotting the pairplot to see the result
sns.pairplot(unsupervised.sample(500))
plt.xticks(rotation = 45)
plt.show()

#initalize a scaler
scaler = StandardScaler()

#scale the data
scaled_features = scaler.fit_transform(unsupervised)

# getting a scaled dataframe
X = pd.DataFrame(scaled_features, columns = unsupervised.columns)

#check out the first few rows
X.head()

#initialize PCA
pca = PCA()

#fit the data
pca.fit(X)

PCA()

PCA()

#graphing the cummulative explained variance plot
plt.figure(figsize=(10,8))

#create range of x ticks labels
feats = np.arange(len(pca.explained_variance_ratio_)) + 1

#plotting the lines
plt.plot(feats, pca.explained_variance_ratio_, '-o', label='Individual component')
plt.plot(feats, np.cumsum(pca.explained_variance_ratio_), '-s', label='Cumulative')

#annotating the graph
plt.ylabel('Proportion of Variance Explained')
plt.xlabel('Principal Component')
plt.xticks(feats)
plt.legend(loc=2)

#showing the graph
plt.show();

np.cumsum(pca.explained_variance_ratio_)

array([0.21879903, 0.37834959, 0.50797121, 0.62830925, 0.71534746,
       0.78332812, 0.84722196, 0.89419062, 0.93015478, 0.96335194,
       0.98015713, 0.9925032 , 0.99938687, 0.99999999, 1.        ])

chosen_components = 9

pca = PCA(n_components = chosen_components)

#fit and transform X to get the principal components from PCA
X_pca = pd.DataFrame(pca.fit_transform(X), 
                 columns=['PC'+str(i) for i in range(1, chosen_components+1)])
X_pca

#plotting PC1 and PC2 and see features' project
visualizer = PCA_viz(
    proj_features=True,
    size = (1920,1080),
    colors= '#D4D4D4'
    )
visualizer.fit_transform(X)
visualizer.show()

<Axes: title={'center': 'Principal Component Plot'}, xlabel='$PC_1$', ylabel='$PC_2$'>

#select a random seed
seed=10

#create a function for faster KMeans modeling:

def KMeansModel(X, k):

    #Initialize the model
    kmeans = KMeans(k, n_init=seed, random_state=seed)

    #Train the Model
    kmeans.fit(X)

    #Evaluate perfomance
    score = davies_bouldin_score(X, kmeans.labels_)

    #return the performance
    return [score, kmeans.labels_]

fig, (ax1, ax2) = plt.subplots(ncols=2,figsize=(12, 4))

#Plotting KMeans without PCA
model = KMeans(random_state=seed, n_init=seed)
visualizer = KElbowVisualizer(model, k=(2,10), timings = False, ax=ax1)
visualizer.fit(X)
visualizer.finalize()

#Plotting KMeans with PCA
model = KMeans(random_state=seed, n_init=seed)
visualizer = KElbowVisualizer(model, k=(2,10), timings = False,  ax=ax2)
visualizer.fit(X_pca)
visualizer.finalize()
ax1.set_title("Without PCA")
ax2.set_title("With PCA");

# Calinski Harabasz

fig, (ax1, ax2) = plt.subplots(ncols=2,figsize=(12, 4))

#Plotting KMeans without PCA
model = KMeans(random_state=seed)
visualizer = KElbowVisualizer(model, k=(2,10), metric='calinski_harabasz', 
                              timings=False, locate_elbow=True, ax=ax1)
visualizer.fit(X)
visualizer.finalize()

#plotting KMeans with PCA
model = KMeans(random_state=seed)
visualizer = KElbowVisualizer(model, k=(2,10), metric='calinski_harabasz', timings=False, locate_elbow=True,  ax=ax2)
visualizer.fit(X_pca)
visualizer.finalize()
ax1.set_title("Without PCA")
ax2.set_title("With PCA");

print('No PCA, K = 5, Davies-Bouldin Index: ', KMeansModel(X, 5)[0])
print('No PCA, K = 2, Davies-Bouldin Index: ', KMeansModel(X, 2)[0])
print('PCA, K = 5, Davies-Bouldin Index: ', KMeansModel(X_pca, 5)[0])
print('PCA, K = 2, Davies-Bouldin Index: ', KMeansModel(X_pca, 2)[0])

No PCA, K = 5, Davies-Bouldin Index:  1.7747775910854127
No PCA, K = 2, Davies-Bouldin Index:  2.08145504187499
PCA, K = 5, Davies-Bouldin Index:  1.6855474445079721
PCA, K = 2, Davies-Bouldin Index:  1.9944492123913333

# defining a function for quicker visualization
def visualizing_cluster(X_results, labels, method):
    
    #initalizing subplots
    fig, ax = plt.subplots(1,3, figsize=(15,7))
    fig.suptitle('Box Plots for Purchase Price and Average Temperature by Cluster, '+ method)

    #selecting price and average temperature
    cols = ['Purchase Price ($thou)','Average Temperature Value', 'Anomaly (1901-2000 base period)']

    #plotting
    for i in range(3):
        sns.boxplot(
            ax = ax[i], 
            data= X_results, 
            y = cols[i], 
            x = labels
        ).set(xlabel = 'Cluster')

    #showing the plot
    plt.tight_layout(pad=2)
    plt.show()    

    #DBSCAN() will give us a lot of clusters so I am turning off the legends here
    if method != 'DBSCAN()':
        legend = 'full'
    else:
        legend = False
    
    #let's plot two scatter plots for average temperature vs. purchase price and anomaly vs. purchase price

    for i in range(2):
        
        #not using a subplot because of run time
        fig = plt.figure(figsize = (15,7))

        #plotting two scatter plots 
        sns.scatterplot(
            data = X_results, 
            x = cols[0], 
            y = cols[i+1], 
            hue = labels,
            legend=legend,
            palette = "deep"
        )

        #showing the plot
        plt.title('Clusters by Purchase Price and ' + cols[i] + ', '+ method)
        plt.show()

# getting the best kmeans
score_kmeans, labels_kmeans = KMeansModel(X_pca, 5)

# creating a results dataframe with labels
X_results = data.copy()

# add a column for KMeans Labels
X_results["labels_kmeans"] = labels_kmeans
X_results.head()

#calling our function to visualize the results
visualizing_cluster(X_results, 'labels_kmeans', 'KMeans')

# we do not recommend running this cell because it requires 1.22 TiB of memory
# depending on the configuration of your laptop/computer, it might result in a MemoryError
# we are only leaving this cell to demonstrate that we tried this model with the entire dataset

# plotting four dendograms, each with different linkage style
style = ['single', 'average', 'complete', 'ward'] 

for i in style:

    # initalizing the dendogram with a linkage style in the list above 
    Z = linkage(X_pca, i)

    # customizing the dendogram
    plt.figure(figsize=(20,10))
    plt.grid(False)
    plt.title(i+' linkage')
    plt.xlabel("index")
    plt.ylabel("distance")

    # plotting a truncated dendogram for less computational time
    dendrogram(Z, leaf_font_size=10, truncate_mode='level',p=5);

---------------------------------------------------------------------------
MemoryError                               Traceback (most recent call last)
Cell In[29], line 10
      7 style = ['single', 'average', 'complete', 'ward'] 
      9 for i in style:
---> 10     Z = linkage(X_pca, i)
     11     plt.figure(figsize=(20,10))
     12     plt.grid(False)

File c:\Users\hieuh\AppData\Local\Programs\Python\Python311\Lib\site-packages\scipy\cluster\hierarchy.py:1024, in linkage(y, method, metric, optimal_ordering)
   1018     if (y.shape[0] == y.shape[1] and np.allclose(np.diag(y), 0) and
   1019             xp.all(y >= 0) and np.allclose(y, y.T)):
   1020         warnings.warn('The symmetric non-negative hollow observation '
   1021                       'matrix looks suspiciously like an uncondensed '
   1022                       'distance matrix',
   1023                       ClusterWarning, stacklevel=2)
-> 1024     y = distance.pdist(y, metric)
   1025     y = xp.asarray(y)
   1026 else:

File c:\Users\hieuh\AppData\Local\Programs\Python\Python311\Lib\site-packages\scipy\spatial\distance.py:2180, in pdist(X, metric, out, **kwargs)
   2178 if metric_info is not None:
   2179     pdist_fn = metric_info.pdist_func
-> 2180     return pdist_fn(X, out=out, **kwargs)
   2181 elif mstr.startswith("test_"):
   2182     metric_info = _TEST_METRICS.get(mstr, None)

MemoryError: Unable to allocate 1.22 TiB for an array with shape (168138235671,) and data type float64

# getting 10,000 samples
X_sample = X_pca.sample(10000, random_state=seed)

# replotting the dendogram with 10,000 samples.

#create a list of linkage types
style = ['single', 'average', 'complete', 'ward']


for i in style:

    #initialize linkage with 10,000 samples and same seed
    Z = linkage(X_sample, i)

    #annotating the dendogram
    plt.figure(figsize=(20,10))
    plt.grid(False)
    plt.title(i+' linkage')
    plt.xlabel("index")
    plt.ylabel("distance")

    #graphing the dendogram (using truncate_mode to only include the top nodes)
    dn = dendrogram(Z, truncate_mode='level', p = 5, leaf_font_size=10, show_contracted=True)
    plt.show()

#initialize linkage with 10,000 samples and same seed
Z = linkage(X_sample, 'ward')

#annotate the plot
plt.figure(figsize=(20,10))
plt.grid(False)
plt.title('Ward Linkage HC with 10,000 Samples')
plt.xlabel("index")
plt.ylabel("distance")

#choosing a cut-off
d = 99

#adding the cut-off line
plt.axhline(y=d, c='k')

#graphing the dendogram (using truncate_mode to only include the top nodes)
dendrogram(Z, leaf_font_size=10, color_threshold=d, truncate_mode= 'level', p = 5)
plt.show()

# initializing an Agglomerative Clustering model with 6 clusters and ward linkage style
hc = AgglomerativeClustering(n_clusters=6,linkage='ward')

#getting X's sample's lables 
labels_hc = hc.fit_predict(X_sample)

#getting the Davies Bouldin Score
hc_score = davies_bouldin_score(X_sample, labels_hc)

#print out the performance
print('Performance: ', hc_score)

Performance:  1.6909172920233886

# creating a results dataframe with labels
X_sample_results = data.sample(10000, random_state = seed).copy()

# add a column for HC Labels
X_sample_results["labels_hc"] = labels_hc
X_sample_results.head()

#calling our function to visualize the results
visualizing_cluster(X_sample_results, 'labels_hc', 'Hierarchical Clustering with Ward Linkage')

# initalize a DBSCAN() model
dbs = DBSCAN()

# fit the data
dbs.fit(X_pca)

#get an intial score
score_dbs = davies_bouldin_score(X_pca, dbs.labels_)

# print the model's performance
print('Performance: ', score_dbs)

Performance:  1.2749899182549822

#initalize the Nearest Neighbors algorithm that finds a point nearest neighbor
neighbors = NearestNeighbors(n_neighbors = 18)

#fit to the data
neighbors.fit(X_pca)

#get the points' distances to their nearest neighbors
distances, indices = neighbors.kneighbors(X_pca)

#order points by their distance to the nearest neighbor
distances = np.sort(distances, axis=0)
distance = distances[:,1]

# plot a line graph of the points sorted by its distance to 
plt.plot(distance)
plt.title('Points Sorted by Distance to the Nearest Point among 18 Neighbors')
plt.show()

#define a function to find epsilon based on a min_samples input

def find_epsilon(minPts):

    epsilon =[]

    #trying different epsilon from 0.45 to 1 in steps of 0.05
    for i in [0.45+i*0.05 for i in range(12)]:

        #initialize the model with chosen min_samples and eps 
        dbs = DBSCAN(min_samples=minPts, eps=i)
        
        #fit to data
        dbs.fit(X_pca)

        #get the davies bouldin score
        score = davies_bouldin_score(X_pca, dbs.labels_)

        #append to list of results
        epsilon.append([i, score])
    
    #get a dataframe of epsilon values and its associated DB Index (the lower the bet)
    return pd.DataFrame(epsilon, columns=['epsilon', 'DB Index'])

#min_samples = 18, takes 92 minutes to run
eps_18 = find_epsilon(18)
eps_18

#min_samples = 19, takes 66 minutes to run
eps_19 = find_epsilon(19)
eps_19

#min_samples = 20, takes 65 minutes to run
eps_20 = find_epsilon(20)
eps_20

#min_samples = 21, takes 64 minutes to run
eps_21 = find_epsilon(21)
eps_21

#plotting DBIndex of diffrent min_samples over increasing epsilon
plt.plot(eps_18['epsilon'], eps_18['DB Index'], label = 'min_samples = 18')
plt.plot(eps_19['epsilon'], eps_19['DB Index'], label = 'min_samples = 19')
plt.plot(eps_20['epsilon'], eps_20['DB Index'], label = 'min_samples = 20')
plt.plot(eps_21['epsilon'], eps_21['DB Index'], label = 'min_samples = 21')

#customize the plot
plt.xlabel('epsilon')
plt.ylabel('Davies-Bouldin Index')
plt.title('DBSCAN() Performance with Increasing Epsilon')
plt.legend()
plt.show()

#takes 13 minutes to run

#getting the best DBSCAN()
dbs = DBSCAN(min_samples = 20, eps = 0.85)

#fit the data
dbs.fit(X_pca)

#get the labels
labels_db = dbs.labels_

#get db scores
score_dbs = davies_bouldin_score(X_pca, labels_db)

#get the labels
X_results['labels_db'] = labels_db

#visualize the results
visualizing_cluster(X_results, 'labels_db', 'DBSCAN()')

# Counting the total number of clusters
n_clusters_ = len(set(labels_db))
print('Number of clusters: ', n_clusters_)

Number of clusters:  103

#see the count of samples in each clusters
sns.barplot(X_results['labels_db'].value_counts().sort_values(ascending=False).head(10))
plt.show()

# only focusing on three majority clusters, what do they look like?
visualizing_cluster(X_results[X_results['labels_db'].isin([-1,1,0])], 'labels_db', '3 major clusters in DBSCAN()')

# denoting all non-dominant clusters as category 2
X_results['labels_db'] = X_results['labels_db'].apply(lambda x: 2 if x not in [-1,1,0] else x)

# define a mapping function
def mapping(data, labels, method):

    for i in [2000, 2008, 2012, 2019]:
        df = data[data['Year']==i]
        fig, ax = plt.subplots(1, 1, figsize=(12, 6))
        ax.set_xlim(-130, -65)
        ax.set_ylim(24, 50)
        ax.set_axis_off()
        ax.set_title(method + ' Clusters by US Counties in ' + str(i))
        us_counties.geometry.boundary.plot(ax=ax, edgecolor='k',linewidth = 0.5)
        df.plot(column=labels, ax=ax, cmap='Oranges_r', legend = True)

mapping(X_results, 'labels_kmeans', 'KMeans')

mapping(X_sample_results, 'labels_hc','HC with Ward')

mapping(X_results, 'labels_db', 'DBSCAN()')

table = pd.DataFrame({
    'Model' : ['KMeans', 'Hierarchical Clustering', 'DBSCAN()'],
    'Dataset': ['All Samples', '10,000 Randomly Selected Samples', 'All Samples'],
    'Tuned Parameters': ['K = 5', 'Ward Linkage', 'min_samples = 20, eps = 0.85'],
    'Scores' : [score_kmeans, hc_score, score_dbs]
})
table

	Date	Year	Month	RegionName	State	Bottom-Tier Average Home Value Estimate	Top-Tier Average Home Value Estimate	Value	Anomaly (1901-2000 base period)	1901-2000 Mean	StateAbbrv	Initial Fees and Charges (%)	Effective Rate (%)	Term to Maturity	Loan Amount ($thou)	Purchase Price ($thou)	Loan-to-Price Ratio (%)	Share of Total Market (%)
0	2000-01-31	2000	1	Los Angeles County	CA	122826.545216	400192.738911	52.3	4.5	47.8	CA	0.93	8.01	15	103.0	173.6	64.7	7.0
1	2000-01-31	2000	1	Los Angeles County	CA	122826.545216	400192.738911	52.3	4.5	47.8	CA	0.81	8.20	30	129.9	166.8	81.1	61.1
2	2000-01-31	2000	1	Cook County	IL	71689.188026	302264.545103	24.9	2.4	22.5	IL	0.93	8.01	15	103.0	173.6	64.7	7.0
3	2000-01-31	2000	1	Cook County	IL	71689.188026	302264.545103	24.9	2.4	22.5	IL	0.81	8.20	30	129.9	166.8	81.1	61.1
4	2000-01-31	2000	1	Harris County	TX	61195.175569	217850.462915	57.4	5.6	51.8	TX	0.93	8.01	15	103.0	173.6	64.7	7.0

	STATEFP	COUNTYFP	COUNTYNS	GEOID	NAME	NAMELSAD	LSAD	CLASSFP	MTFCC	CSAFP	CBSAFP	METDIVFP	FUNCSTAT	ALAND	AWATER	INTPTLAT	INTPTLON	geometry
0	31	039	00835841	31039	Cuming	Cuming County	06	H1	G4020	None	None	None	A	1477644346	10691216	+41.9158651	-096.7885168	POLYGON ((-96.55515 41.91587, -96.55515 41.914...
1	53	069	01513275	53069	Wahkiakum	Wahkiakum County	06	H1	G4020	None	None	None	A	680980770	61564427	+46.2946377	-123.4244583	POLYGON ((-123.72755 46.26449, -123.72755 46.2...
2	35	011	00933054	35011	De Baca	De Baca County	06	H1	G4020	None	None	None	A	6016818946	29090018	+34.3592729	-104.3686961	POLYGON ((-104.89337 34.08894, -104.89337 34.0...
3	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	None	None	A	2169272978	22847034	+40.7835474	-096.6886584	POLYGON ((-96.68493 40.52330, -96.69219 40.523...
4	31	129	00835886	31129	Nuckolls	Nuckolls County	06	H1	G4020	None	None	None	A	1489645185	1718484	+40.1764918	-098.0468422	POLYGON ((-98.27370 40.11840, -98.27374 40.122...

	STATEFP	COUNTYFP	COUNTYNS	GEOID	NAME	NAMELSAD	LSAD	CLASSFP	MTFCC	CSAFP	...	1901-2000 Mean	StateAbbrv	Initial Fees and Charges (%)	Effective Rate (%)	Term to Maturity	Loan Amount ($thou)	Purchase Price ($thou)	Loan-to-Price Ratio (%)	Share of Total Market (%)	StateFPS
0	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	...	22.2	NE	0.93	8.01	15	103.0	173.6	64.7	7.0	31
1	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	...	22.2	NE	0.81	8.20	30	129.9	166.8	81.1	61.1	31
2	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	...	27.3	NE	0.91	8.18	15	100.5	172.8	63.1	7.2	31
3	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	...	27.3	NE	0.78	8.39	30	127.8	163.7	81.2	58.6	31
4	31	109	00835876	31109	Lancaster	Lancaster County	06	H1	G4020	None	...	38.0	NE	0.87	8.24	15	101.2	176.9	62.7	7.2	31

	1901-2000 Mean	ALAND	AWATER	Anomaly (1901-2000 base period)	Average Temperature Value	Bottom-Tier Average Home Value Estimate	Effective Rate (%)	INTPTLAT	INTPTLON	Initial Fees and Charges (%)	Loan Amount ($thou)	Loan-to-Price Ratio (%)	Month	Purchase Price ($thou)	Share of Total Market (%)	Term to Maturity	Top-Tier Average Home Value Estimate	Year
0	22.2	2169272978	22847034	6.6	28.8	77619.405010	8.01	+40.7835474	-096.6886584	0.93	103.0	64.7	1	173.6	7.0	15	202516.204425	2000
1	22.2	2169272978	22847034	6.6	28.8	77619.405010	8.20	+40.7835474	-096.6886584	0.81	129.9	81.1	1	166.8	61.1	30	202516.204425	2000
2	27.3	2169272978	22847034	9.4	36.7	77754.652496	8.18	+40.7835474	-096.6886584	0.91	100.5	63.1	2	172.8	7.2	15	202854.350975	2000
3	27.3	2169272978	22847034	9.4	36.7	77754.652496	8.39	+40.7835474	-096.6886584	0.78	127.8	81.2	2	163.7	58.6	30	202854.350975	2000
4	38.0	2169272978	22847034	6.5	44.5	77886.535500	8.24	+40.7835474	-096.6886584	0.87	101.2	62.7	3	176.9	7.2	15	203164.680964	2000

	1901-2000 Mean	ALAND	AWATER	Anomaly (1901-2000 base period)	Average Temperature Value	Bottom-Tier Average Home Value Estimate	Effective Rate (%)	INTPTLAT	INTPTLON	Initial Fees and Charges (%)	Loan Amount ($thou)	Loan-to-Price Ratio (%)	Month	Purchase Price ($thou)	Share of Total Market (%)	Term to Maturity	Top-Tier Average Home Value Estimate	Year
count	579894.000000	5.798940e+05	5.798940e+05	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	579894.000000	5.798940e+05	579894.000000
mean	54.317547	2.338742e+09	1.819740e+08	1.385632	55.702485	94103.848741	4.955709	37.986718	-89.613755	0.891515	177.251586	72.512364	6.434317	260.126543	37.364843	22.500000	2.851340e+05	2010.712761
std	17.099949	3.703393e+09	5.523955e+08	3.388082	17.135498	64177.827177	1.232351	4.615860	12.635366	0.365064	36.994047	8.549974	3.466285	36.813915	29.181675	7.500006	2.108634e+05	5.361191
min	1.800000	5.868388e+07	9.251200e+04	-22.200000	-5.200000	6109.396309	2.890000	25.586120	-124.210929	0.180000	60.300000	42.300000	1.000000	124.000000	1.300000	15.000000	5.947229e+04	2000.000000
25%	41.300000	1.055922e+09	1.047176e+07	-0.700000	42.800000	52879.469613	3.980000	34.763109	-94.888456	0.540000	150.900000	67.100000	3.000000	237.700000	7.700000	15.000000	1.884248e+05	2007.000000
50%	55.500000	1.479110e+09	2.694270e+07	1.300000	57.200000	77205.031646	4.650000	38.423084	-86.034263	0.910000	179.800000	74.800000	6.000000	262.500000	32.450000	22.500000	2.347503e+05	2011.000000
75%	68.800000	2.206699e+09	8.116171e+07	3.400000	70.000000	115764.095664	5.960000	41.355319	-81.400955	1.220000	204.400000	80.300000	9.000000	286.100000	65.500000	30.000000	3.139183e+05	2015.000000
max	91.700000	5.197631e+10	1.405367e+10	19.800000	96.200000	915009.312632	8.550000	48.842653	-68.666616	1.510000	254.800000	83.500000	12.000000	347.300000	82.900000	30.000000	5.178158e+06	2019.000000

Modeling Climate and Housing Data¶

Question 3: Are there any hidden associations between housing prices and average temperature or temperature anomalies in this dataset that is worth exploring?¶

90-803 Machine Learning Foundations with Python (Spring 2024)¶

Team 17: Yoko, Tessa, Hannah¶

Importing the Relevant Libraries¶

Read in the dataset and Merge with Geodata¶

Initial Cleaning¶

Exploration Conclusion:¶

Feature Engineering:¶

(1) Dropping Categorical Values¶

(2) Scaling the Dataset¶

(3) Principal Component Analysis¶

Model #1: KMeans Clustering¶

KMeans Function¶

Tuning: Finding the Best K¶

Visualizing Results for the Best KMeans¶

Model #2: Hierarchical Clustering¶

Finding the best number of clusters¶

Conclusions¶

Modeling¶

Visualizing the Results¶

Model #3: DBSCAN¶

Default Model:¶

Tuning:¶

Mapping the Clusters on a US County Map¶

Conclusion:¶

References:¶

	1901-2000 Mean	ALAND	AWATER	Anomaly (1901-2000 base period)	Average Temperature Value	Bottom-Tier Average Home Value Estimate	Effective Rate (%)	INTPTLAT	INTPTLON	Initial Fees and Charges (%)	Loan Amount ($thou)	Loan-to-Price Ratio (%)	Purchase Price ($thou)	Share of Total Market (%)	Top-Tier Average Home Value Estimate
0	-1.878226	-0.04576	-0.288067	1.539034	-1.569987	-0.256856	2.478430	0.605918	-0.559929	0.105420	-2.007124	-0.913730	-2.350378	-1.040546	-0.391808
1	-1.878226	-0.04576	-0.288067	1.539034	-1.569987	-0.256856	2.632607	0.605918	-0.559929	-0.223290	-1.279980	1.004406	-2.535091	0.813359	-0.391808
2	-1.579980	-0.04576	-0.288067	2.365461	-1.108955	-0.254749	2.616377	0.605918	-0.559929	0.050635	-2.074703	-1.100866	-2.372109	-1.033692	-0.390204
3	-1.579980	-0.04576	-0.288067	2.365461	-1.108955	-0.254749	2.786784	0.605918	-0.559929	-0.305468	-1.336746	1.016102	-2.619298	0.727689	-0.390204
4	-0.954246	-0.04576	-0.288067	1.509519	-0.653759	-0.252694	2.665065	0.605918	-0.559929	-0.058935	-2.055781	-1.147649	-2.260738	-1.033692	-0.388732

	PC1	PC2	PC3	PC4	PC5	PC6	PC7	PC8	PC9
0	-3.289982	2.388260	-1.198709	-0.055656	0.564186	1.777334	-0.790117	0.412116	-1.293996
1	-1.989475	2.051385	-2.736585	1.713475	0.567079	1.836836	-0.845985	0.418021	-1.234150
2	-3.450215	2.027563	-0.864623	0.075168	0.499991	2.573917	-1.031670	0.270344	-1.159624
3	-2.136587	1.681973	-2.450586	1.902305	0.506540	2.644255	-1.091122	0.273252	-1.163395
4	-3.552593	1.424556	-0.604358	0.265270	0.534005	1.759478	-0.817116	-0.107692	-1.013886
...	...	...	...	...	...	...	...	...	...
579889	2.376940	0.195019	-1.725089	-0.526952	-0.236777	1.357094	-0.450669	0.580186	0.850816
579890	1.074559	0.203717	0.078919	-2.607085	-0.125588	-1.045493	0.271752	0.224959	1.100795
579891	2.557303	-0.196038	-1.774581	-0.456729	-0.121275	-0.947583	0.194310	0.230870	1.126430
579892	1.073780	-0.549874	0.400579	-2.113795	-0.179681	0.505716	-0.166365	-0.067852	0.567405
579893	2.498660	-0.903780	-1.180729	-0.282335	-0.199457	0.531939	-0.223249	-0.053049	0.908494

	STATEFP	COUNTYFP	COUNTYNS	GEOID	NAME	NAMELSAD	LSAD	CLASSFP	MTFCC	FUNCSTAT	...	StateAbbrv	Initial Fees and Charges (%)	Effective Rate (%)	Term to Maturity	Loan Amount ($thou)	Purchase Price ($thou)	Loan-to-Price Ratio (%)	Share of Total Market (%)	StateFPS	labels_hc
473260	39	097	01074061	39097	Madison	Madison County	06	H1	G4020	A	...	OH	1.08	3.99	15	211.5	338.9	65.0	6.9	39	3
91859	51	045	01673664	51045	Craig	Craig County	06	H1	G4020	A	...	VA	1.31	4.70	30	209.3	265.6	81.4	62.7	51	4
55272	16	053	00395662	16053	Jerome	Jerome County	06	H1	G4020	A	...	ID	0.49	4.85	15	157.2	254.4	68.3	10.2	16	0
572927	33	001	00873174	33001	Belknap	Belknap County	06	H1	G4020	A	...	NH	0.64	6.26	30	190.4	251.9	78.0	80.7	33	5
113911	08	037	00198134	08037	Eagle	Eagle County	06	H1	G4020	A	...	CO	0.70	8.05	30	137.2	179.3	79.9	72.6	08	0

	epsilon	DB Index
0	0.45	1.473271
1	0.50	1.475011
2	0.55	1.348007
3	0.60	1.442000
4	0.65	1.376603
5	0.70	1.417793
6	0.75	1.397424
7	0.80	1.318577
8	0.85	1.335645
9	0.90	1.390239
10	0.95	1.397140
11	1.00	1.524045

	epsilon	DB Index
0	0.45	1.475985
1	0.50	1.500787
2	0.55	1.358402
3	0.60	1.412694
4	0.65	1.370813
5	0.70	1.451285
6	0.75	1.408073
7	0.80	1.344550
8	0.85	1.291687
9	0.90	1.425404
10	0.95	1.364468
11	1.00	1.528175

	epsilon	DB Index
0	0.45	1.486222
1	0.50	1.473333
2	0.55	1.356084
3	0.60	1.424062
4	0.65	1.401325
5	0.70	1.439877
6	0.75	1.418717
7	0.80	1.365897
8	0.85	1.275952
9	0.90	1.428955
10	0.95	1.379657
11	1.00	1.530542

	epsilon	DB Index
0	0.45	1.503944
1	0.50	1.454713
2	0.55	1.402328
3	0.60	1.413953
4	0.65	1.425687
5	0.70	1.416320
6	0.75	1.426620
7	0.80	1.394603
8	0.85	1.314492
9	0.90	1.389012
10	0.95	1.384154
11	1.00	1.502831

	Model	Dataset	Tuned Parameters	Scores
0	KMeans	All Samples	K = 5	1.685586
1	Hierarchical Clustering	10,000 Randomly Selected Samples	Ward Linkage	1.690917
2	DBSCAN()	All Samples	min_samples = 20, eps = 0.85	1.275934